Importing packages¶
In [ ]:
import numpy as np
import matplotlib.pyplot as plt
import os
import tensorflow as tf
from PIL import Image
import random
import numpy as np
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D
from tensorflow.keras.applications import VGG16
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.preprocessing.image import ImageDataGenerator, img_to_array, load_img
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping
from sklearn.metrics import confusion_matrix
from tensorflow.keras.layers import Dense, Flatten, Dropout
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
from tensorflow.keras.models import load_model
from sklearn.metrics import classification_report, precision_recall_curve, auc
1. Obtain the Data : Dogs vs Cats dataset¶
In [ ]:
base_dir = os.path.join(os.getcwd(), 'data\kaggle_dogs_vs_cats_small')
train_dir = os.path.join(base_dir, 'train')
validation_dir = os.path.join(base_dir, 'validation')
test_dir = os.path.join(base_dir, 'test')
if not os.path.exists(base_dir):
print("Base directory does not exist!")
else:
def count_images(directory):
total_images = 0
for category in os.listdir(directory):
category_path = os.path.join(directory, category)
if os.path.isdir(category_path):
total_images += len(os.listdir(category_path))
return total_images
total_train_images = count_images(train_dir)
total_validation_images = count_images(validation_dir)
total_test_images = count_images(test_dir)
print(f"Total training images: {total_train_images}")
print(f"Total validation images: {total_validation_images}")
print(f"Total test images: {total_test_images}")
Total training images: 2000 Total validation images: 1000 Total test images: 2000
In [ ]:
def visualize_class_distribution(data_dir, set_name):
categories = ['cat', 'dog']
image_counts = [len(os.listdir(os.path.join(data_dir, category))) for category in categories]
total_images = sum(image_counts)
# Create the bar chart
plt.figure(figsize=(8, 6))
bars = plt.bar(categories, image_counts, color=['lightblue', 'lightcoral'], edgecolor='black', alpha=0.85)
plt.title(f'{set_name} Class Distribution', fontsize=16, fontweight='bold')
plt.xlabel('Class', fontsize=12)
plt.ylabel('Image Count', fontsize=12)
for bar, count in zip(bars, image_counts):
percent = (count / total_images) * 100
plt.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 3,
f'{count}\n({percent:.1f}%)', ha='center', va='bottom', fontsize=10)
plt.grid(axis='y', linestyle=':', linewidth=0.8, alpha=0.6)
plt.ylim(0, max(image_counts) + 100)
plt.tight_layout()
plt.show()
visualize_class_distribution(train_dir, 'Training Data')
visualize_class_distribution(validation_dir, 'Validation Data')
visualize_class_distribution(test_dir, 'Test Data')
The number of images for each class is the same across all subsets, indicating that the dataset is balanced.
2.2 Image Dimensions Analysis¶
To assess variations in image size, we analyzed the width and height distributions of the images.
In [ ]:
def inspect_image_sizes(data_dir):
img_widths, img_heights = [], []
for category in ['cat', 'dog']:
category_path = os.path.join(data_dir, category)
for img_file in os.listdir(category_path):
img_path = os.path.join(category_path, img_file)
with Image.open(img_path) as img:
img_widths.append(img.size[0])
img_heights.append(img.size[1])
# Plot histograms
plt.figure(figsize=(10, 6))
plt.hist(img_widths, bins=20, alpha=0.6, color='cornflowerblue', label='Image Widths')
plt.hist(img_heights, bins=20, alpha=0.6, color='lightcoral', label='Image Heights')
plt.title('Distribution of Image Dimensions', fontsize=16, fontweight='bold')
plt.xlabel('Pixel Count', fontsize=12)
plt.ylabel('Number of Images', fontsize=12)
plt.legend()
plt.grid(axis='y', linestyle='--', alpha=0.7)
plt.tight_layout()
plt.show()
inspect_image_sizes(train_dir)
- Image sizes vary considerably.
- To ensure consistent input dimensions for the neural network, image sizes need to be standardized.
3.3 Sample Images Visualization¶
To visually inspect the dataset for quality, diversity, and potential issues, we displayed a sample of images from both the dog and cat categories.
In [ ]:
def show_sample_images(data_dir, label, num_images=5):
label_path = os.path.join(data_dir, label)
selected_images = random.sample(os.listdir(label_path), num_images)
plt.figure(figsize=(16, 4))
for idx, image_name in enumerate(selected_images):
image_path = os.path.join(label_path, image_name)
image = load_img(image_path) # Load the image
plt.subplot(1, num_images, idx + 1) # Create subplot
plt.imshow(image)
plt.title(f"{label.title()}", fontsize=12, fontweight='bold')
plt.axis('off')
plt.suptitle(f"Sample Images: {label.title()}", fontsize=16, fontweight='bold', y=1.05)
plt.tight_layout()
plt.show()
show_sample_images(train_dir, 'cat', num_images=5)
show_sample_images(train_dir, 'dog', num_images=5)
- Most of the images are high-quality, with each class exhibiting distinct characteristics.
- Some images may benefit from data augmentation due to background noise, which could enhance the model’s robustness.
2.4 Augmented Data Visualization¶
To improve the model’s ability to generalize, we applied augmentations (e.g., rotation, zoom, and shifts) to introduce additional variability to the dataset.
In [ ]:
def visualize_augmentations(image_path, augmentations, num_samples=5):
data_generator = ImageDataGenerator(**augmentations)
image = load_img(image_path)
image_array = img_to_array(image)
image_array = np.expand_dims(image_array, axis=0)
plt.figure(figsize=(16, 4))
for index, augmented in enumerate(data_generator.flow(image_array, batch_size=1)):
if index >= num_samples:
break
plt.subplot(1, num_samples, index + 1)
plt.imshow(augmented[0].astype('uint8'))
plt.axis('off')
plt.title(f"Augmented #{index + 1}", fontsize=10)
plt.suptitle("Augmented Image Samples", fontsize=14, fontweight='bold', y=1.05)
plt.tight_layout()
plt.show()
augmentations = {
'rotation_range': 40,
'width_shift_range': 0.2,
'height_shift_range': 0.2,
'shear_range': 0.2,
'zoom_range': 0.2,
'horizontal_flip': True,
'fill_mode': 'nearest'
}
sample_cat_image = os.path.join(train_dir, 'cat', os.listdir(os.path.join(train_dir, 'cat'))[12])
visualize_augmentations(sample_cat_image, augmentations, num_samples=5)
- Augmented images show notable alterations, such as flips and rotations.
- The augmentation process successfully introduces variability without altering the key features of each class.
3) Train two networks (use callbacks to save the best model version):¶
- Define a Neural Network of your choice.
- Fine-Tune VGG16 (pre-trained on imagenet). Make sure to use validation to test for over-fitting. Plot the appropriate graph.
Load Dataset¶
In [ ]:
IMAGE_DIMS = (150, 150) # Height and width of input images
BATCH_SIZE = 32 # Number of samples per batch
def create_data_generator(rescale_factor, augmentations=None):
if augmentations:
return ImageDataGenerator(rescale=rescale_factor, **augmentations)
return ImageDataGenerator(rescale=rescale_factor)
# Augmentations for training data
train_augmentations = {
'rotation_range': 40,
'width_shift_range': 0.2,
'height_shift_range': 0.2,
'shear_range': 0.2,
'zoom_range': 0.2,
'horizontal_flip': True
}
train_data_gen = create_data_generator(rescale_factor=1.0 / 255, augmentations=train_augmentations)
validation_data_gen = create_data_generator(rescale_factor=1.0 / 255)
test_data_gen = create_data_generator(rescale_factor=1.0 / 255)
def prepare_data_generator(generator, directory, dataset_type, shuffle=True):
return generator.flow_from_directory(
directory,
target_size=IMAGE_DIMS,
batch_size=BATCH_SIZE,
class_mode='binary',
shuffle=shuffle,
subset=None
)
train_generator = prepare_data_generator(train_data_gen, train_dir, "Training Data")
validation_generator = prepare_data_generator(validation_data_gen, validation_dir, "Validation Data")
test_generator = prepare_data_generator(test_data_gen, test_dir, "Test Data", shuffle=False)
Found 2000 images belonging to 2 classes. Found 1000 images belonging to 2 classes. Found 2000 images belonging to 2 classes.
Define and Train the Model¶
In [ ]:
IMG_HEIGHT, IMG_WIDTH = IMAGE_DIMS # Extract height and width from the tuple
def build_cnn(input_shape):
model = Sequential([
Conv2D(filters=32, kernel_size=(3, 3), activation='relu', input_shape=input_shape),
MaxPooling2D(pool_size=(2, 2)),
Conv2D(filters=64, kernel_size=(3, 3), activation='relu'),
MaxPooling2D(pool_size=(2, 2)),
Conv2D(filters=128, kernel_size=(3, 3), activation='relu'),
MaxPooling2D(pool_size=(2, 2)),
Flatten(),
Dense(units=512, activation='relu'),
Dropout(rate=0.5),
Dense(units=1, activation='sigmoid')
])
return model
def compile_model(model):
model.compile(
optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy']
)
def create_callbacks():
return [
ModelCheckpoint(filepath='best_model_checkpoint.keras', save_best_only=True, monitor='val_loss'),
EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)
]
def train_model(model, train_data, validation_data, epochs, callbacks):
return model.fit(
train_data,
validation_data=validation_data,
epochs=epochs,
callbacks=callbacks
)
def evaluate_model(model, test_data):
test_loss, test_accuracy = model.evaluate(test_data)
print(f"Test Accuracy: {test_accuracy * 100:.2f}%")
print(f"Test Loss: {test_loss:.4f}")
input_shape = (*IMAGE_DIMS, 3)
cnn_model = build_cnn(input_shape)
compile_model(cnn_model)
model_callbacks = create_callbacks()
# Train the model
model_history = train_model(
model=cnn_model,
train_data=train_generator,
validation_data=validation_generator,
epochs=20,
callbacks=model_callbacks
)
# Test the model
evaluate_model(cnn_model, test_generator)
C:\Users\rohit\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\LocalCache\local-packages\Python311\site-packages\keras\src\trainers\data_adapters\py_dataset_adapter.py:121: UserWarning: Your `PyDataset` class should call `super().__init__(**kwargs)` in its constructor. `**kwargs` can include `workers`, `use_multiprocessing`, `max_queue_size`. Do not pass these arguments to `fit()`, as they will be ignored. self._warn_if_super_not_called()
Epoch 1/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 88s 1s/step - accuracy: 0.5172 - loss: 0.9013 - val_accuracy: 0.5000 - val_loss: 0.6992 Epoch 2/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 36s 565ms/step - accuracy: 0.5228 - loss: 0.6867 - val_accuracy: 0.5320 - val_loss: 0.6944 Epoch 3/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 40s 640ms/step - accuracy: 0.5896 - loss: 0.6610 - val_accuracy: 0.5820 - val_loss: 0.6673 Epoch 4/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 25s 392ms/step - accuracy: 0.6299 - loss: 0.6462 - val_accuracy: 0.6330 - val_loss: 0.6423 Epoch 5/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 24s 378ms/step - accuracy: 0.6227 - loss: 0.6542 - val_accuracy: 0.6260 - val_loss: 0.6530 Epoch 6/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 25s 400ms/step - accuracy: 0.6521 - loss: 0.6505 - val_accuracy: 0.6450 - val_loss: 0.6502 Epoch 7/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 24s 387ms/step - accuracy: 0.6157 - loss: 0.6623 - val_accuracy: 0.5900 - val_loss: 0.6780 Epoch 8/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 25s 401ms/step - accuracy: 0.6598 - loss: 0.6333 - val_accuracy: 0.6280 - val_loss: 0.6189 Epoch 9/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 25s 399ms/step - accuracy: 0.6532 - loss: 0.6124 - val_accuracy: 0.5850 - val_loss: 0.6846 Epoch 10/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 26s 408ms/step - accuracy: 0.6126 - loss: 0.6592 - val_accuracy: 0.6830 - val_loss: 0.5868 Epoch 11/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 25s 401ms/step - accuracy: 0.6907 - loss: 0.5940 - val_accuracy: 0.6920 - val_loss: 0.5786 Epoch 12/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 25s 390ms/step - accuracy: 0.6868 - loss: 0.5890 - val_accuracy: 0.6480 - val_loss: 0.6270 Epoch 13/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 24s 372ms/step - accuracy: 0.6712 - loss: 0.6108 - val_accuracy: 0.6700 - val_loss: 0.6264 Epoch 14/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 25s 393ms/step - accuracy: 0.6908 - loss: 0.5884 - val_accuracy: 0.6950 - val_loss: 0.5601 Epoch 15/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 23s 371ms/step - accuracy: 0.7067 - loss: 0.5584 - val_accuracy: 0.7140 - val_loss: 0.6205 Epoch 16/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 24s 386ms/step - accuracy: 0.7076 - loss: 0.5691 - val_accuracy: 0.7180 - val_loss: 0.5471 Epoch 17/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 24s 386ms/step - accuracy: 0.7072 - loss: 0.5725 - val_accuracy: 0.7300 - val_loss: 0.5325 Epoch 18/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 24s 380ms/step - accuracy: 0.7322 - loss: 0.5459 - val_accuracy: 0.7230 - val_loss: 0.5507 Epoch 19/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 23s 359ms/step - accuracy: 0.7221 - loss: 0.5617 - val_accuracy: 0.7150 - val_loss: 0.5333 Epoch 20/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 23s 358ms/step - accuracy: 0.7112 - loss: 0.5530 - val_accuracy: 0.6930 - val_loss: 0.5850 63/63 ━━━━━━━━━━━━━━━━━━━━ 6s 90ms/step - accuracy: 0.7412 - loss: 0.5830 Test Accuracy: 71.75% Test Loss: 0.5605
Accuracy and Loss Curves¶
In [ ]:
def plot_training_curves_custom(history):
epochs = range(1, len(history.history['accuracy']) + 1)
# Accuracy Plot
plt.figure(figsize=(10, 5))
plt.plot(epochs, history.history['accuracy'], marker='o', linestyle='-', label='Training Accuracy')
plt.plot(epochs, history.history['val_accuracy'], marker='s', linestyle='--', label='Validation Accuracy')
plt.title('Training and Validation Accuracy (Custom Model)', fontsize=14)
plt.xlabel('Epochs', fontsize=12)
plt.ylabel('Accuracy', fontsize=12)
plt.legend()
plt.grid(True, linestyle='--', alpha=0.5)
plt.show()
# Loss Plot
plt.figure(figsize=(10, 5))
plt.plot(epochs, history.history['loss'], marker='o', linestyle='-', label='Training Loss')
plt.plot(epochs, history.history['val_loss'], marker='s', linestyle='--', label='Validation Loss')
plt.title('Training and Validation Loss (Custom Model)', fontsize=14)
plt.xlabel('Epochs', fontsize=12)
plt.ylabel('Loss', fontsize=12)
plt.legend()
plt.grid(True, linestyle='--', alpha=0.5)
plt.show()
plot_training_curves_custom(model_history)
Fine-Tune VGG16¶
In [ ]:
vgg_base_model = VGG16(weights='imagenet', include_top=False, input_shape=(IMG_HEIGHT, IMG_WIDTH, 3))
for layer in vgg_base_model.layers:
layer.trainable = False
x = vgg_base_model.output
x = Flatten()(x)
x = Dense(256, activation='relu')(x)
x = Dropout(0.5)(x)
final_output = Dense(1, activation='sigmoid')(x)
custom_vgg_model = Model(inputs=vgg_base_model.input, outputs=final_output)
custom_vgg_model.compile(optimizer=Adam(learning_rate=0.0001), loss='binary_crossentropy', metrics=['accuracy'])
model_callbacks = [
ModelCheckpoint(filepath='best_vgg_model.keras', save_best_only=True, monitor='val_loss'),
EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)
]
training_history = custom_vgg_model.fit(
train_generator,
validation_data=validation_generator,
epochs=20,
callbacks=model_callbacks
)
test_loss, test_acc = custom_vgg_model.evaluate(test_generator)
print(f"Test Accuracy: {test_acc * 100:.2f}%")
print(f"Test Loss: {test_loss:.4f}")
Epoch 1/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 133s 2s/step - accuracy: 0.6138 - loss: 0.6966 - val_accuracy: 0.8560 - val_loss: 0.3644 Epoch 2/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 136s 2s/step - accuracy: 0.7562 - loss: 0.4788 - val_accuracy: 0.8610 - val_loss: 0.3230 Epoch 3/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 133s 2s/step - accuracy: 0.7980 - loss: 0.4139 - val_accuracy: 0.8820 - val_loss: 0.2903 Epoch 4/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 135s 2s/step - accuracy: 0.8117 - loss: 0.3961 - val_accuracy: 0.8880 - val_loss: 0.2794 Epoch 5/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 134s 2s/step - accuracy: 0.8179 - loss: 0.3876 - val_accuracy: 0.8800 - val_loss: 0.2793 Epoch 6/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 136s 2s/step - accuracy: 0.8365 - loss: 0.3692 - val_accuracy: 0.8910 - val_loss: 0.2665 Epoch 7/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 118s 2s/step - accuracy: 0.8397 - loss: 0.3605 - val_accuracy: 0.8910 - val_loss: 0.2652 Epoch 8/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 110s 2s/step - accuracy: 0.8548 - loss: 0.3294 - val_accuracy: 0.8900 - val_loss: 0.2606 Epoch 9/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 111s 2s/step - accuracy: 0.8516 - loss: 0.3497 - val_accuracy: 0.8930 - val_loss: 0.2526 Epoch 10/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 112s 2s/step - accuracy: 0.8562 - loss: 0.3337 - val_accuracy: 0.8910 - val_loss: 0.2504 Epoch 11/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 110s 2s/step - accuracy: 0.8507 - loss: 0.3520 - val_accuracy: 0.8960 - val_loss: 0.2530 Epoch 12/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 110s 2s/step - accuracy: 0.8639 - loss: 0.3200 - val_accuracy: 0.8970 - val_loss: 0.2501 Epoch 13/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 110s 2s/step - accuracy: 0.8518 - loss: 0.3244 - val_accuracy: 0.8900 - val_loss: 0.2630 Epoch 14/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 109s 2s/step - accuracy: 0.8562 - loss: 0.3249 - val_accuracy: 0.8960 - val_loss: 0.2508 Epoch 15/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 111s 2s/step - accuracy: 0.8510 - loss: 0.3297 - val_accuracy: 0.8990 - val_loss: 0.2479 Epoch 16/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 111s 2s/step - accuracy: 0.8690 - loss: 0.3054 - val_accuracy: 0.9060 - val_loss: 0.2407 Epoch 17/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 110s 2s/step - accuracy: 0.8748 - loss: 0.2864 - val_accuracy: 0.9060 - val_loss: 0.2363 Epoch 18/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 112s 2s/step - accuracy: 0.8596 - loss: 0.3126 - val_accuracy: 0.8960 - val_loss: 0.2443 Epoch 19/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 110s 2s/step - accuracy: 0.8726 - loss: 0.3095 - val_accuracy: 0.8900 - val_loss: 0.2493 Epoch 20/20 63/63 ━━━━━━━━━━━━━━━━━━━━ 109s 2s/step - accuracy: 0.8628 - loss: 0.3054 - val_accuracy: 0.8930 - val_loss: 0.2497 63/63 ━━━━━━━━━━━━━━━━━━━━ 74s 1s/step - accuracy: 0.8959 - loss: 0.2500 Test Accuracy: 89.25% Test Loss: 0.2474
VGG16 Fine-Tuning: Accuracy and Loss Curves¶
In [ ]:
def plot_training_metrics_vgg(model_history):
epochs_range = range(1, len(model_history.history['accuracy']) + 1)
# Plot for Accuracy
plt.figure(figsize=(12, 6))
plt.plot(epochs_range, model_history.history['accuracy'], label='Training Accuracy', color='mediumblue', linewidth=2)
plt.plot(epochs_range, model_history.history['val_accuracy'], label='Validation Accuracy', color='forestgreen', linestyle='--', linewidth=2)
plt.fill_between(epochs_range, model_history.history['accuracy'], model_history.history['val_accuracy'], color='lightblue', alpha=0.4)
plt.title('Accuracy Comparison (VGG16 Model)', fontsize=14)
plt.xlabel('Epochs', fontsize=12)
plt.ylabel('Accuracy', fontsize=12)
plt.legend(loc='best')
plt.grid(True)
plt.show()
# Plot for Loss
plt.figure(figsize=(12, 6))
plt.plot(epochs_range, model_history.history['loss'], label='Training Loss', color='firebrick', linewidth=2)
plt.plot(epochs_range, model_history.history['val_loss'], label='Validation Loss', color='darkorange', linestyle='--', linewidth=2)
plt.fill_between(epochs_range, model_history.history['loss'], model_history.history['val_loss'], color='lightcoral', alpha=0.4)
plt.title('Loss Comparison (VGG16 Model)', fontsize=14)
plt.xlabel('Epochs', fontsize=12)
plt.ylabel('Loss', fontsize=12)
plt.legend(loc='best')
plt.grid(True)
plt.show()
plot_training_metrics_vgg(training_history)
4. Explore the relative performance of the models (make sure to load the best version of each model)
4.1 accuracy
4.2 confusion metric
4.3 precision, recall, F1-score
4.4 precision-recall curve
4.5 Explore specific examples in which the model failed to predict correctly.
Load the best models¶
In [ ]:
custom_model = load_model('best_model_checkpoint.keras')
vgg_model = load_model('best_vgg_model.keras')
4.1 Accuracy¶
In [ ]:
# Evaluate the Custom model on the test set
custom_test_loss, custom_test_accuracy = custom_model.evaluate(test_generator)
models = ['Custom Model', 'VGG16 Model']
accuracies = [custom_test_accuracy * 100, test_acc * 100]
# Plotting the bar chart
plt.figure(figsize=(8, 6))
bars = plt.bar(models, accuracies, color=['lightcoral', 'mediumseagreen'], edgecolor='black', linewidth=1.5)
for bar, accuracy in zip(bars, accuracies):
plt.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 2, f"{accuracy:.2f}%",
ha='center', va='bottom', color='black', fontsize=12, fontweight='bold')
plt.ylim(0, 110)
plt.title("Comparison of Test Accuracy Between Models", fontsize=16, fontweight='bold')
plt.xlabel("Models", fontsize=14)
plt.ylabel("Accuracy (%)", fontsize=14)
plt.grid(axis='y', linestyle='-.', alpha=0.6)
plt.show()
63/63 ━━━━━━━━━━━━━━━━━━━━ 5s 81ms/step - accuracy: 0.7412 - loss: 0.5830
4.2 Confusion Matrix¶
In [ ]:
custom_preds = (custom_model.predict(test_generator) > 0.5).astype(int)
vgg_preds = (vgg_model.predict(test_generator) > 0.5).astype(int)
true_labels = test_generator.classes
# Confusion Matrix for Custom Model
custom_cm = confusion_matrix(true_labels, custom_preds)
fig, ax = plt.subplots(figsize=(7, 5))
ConfusionMatrixDisplay(custom_cm, display_labels=['Cat', 'Dog']).plot(cmap='Blues', ax=ax)
plt.title("Custom Model Confusion Matrix", fontsize=16, fontweight='bold')
plt.xlabel('Predicted Labels', fontsize=12)
plt.ylabel('True Labels', fontsize=12)
plt.show()
# Confusion Matrix for VGG16 Model
vgg_cm = confusion_matrix(true_labels, vgg_preds)
fig, ax = plt.subplots(figsize=(7, 5))
ConfusionMatrixDisplay(vgg_cm, display_labels=['Cat', 'Dog']).plot(cmap='Greens', ax=ax)
plt.title("VGG16 Model Confusion Matrix", fontsize=16, fontweight='bold')
plt.xlabel('Predicted Labels', fontsize=12)
plt.ylabel('True Labels', fontsize=12)
plt.show()
63/63 ━━━━━━━━━━━━━━━━━━━━ 4s 69ms/step 63/63 ━━━━━━━━━━━━━━━━━━━━ 86s 1s/step
4.3 Precision, Recall, F1-Score¶
In [ ]:
print("\n" + "="*50)
print("Custom Model Classification Report")
custom_report = classification_report(true_labels, custom_preds, target_names=['Cat', 'Dog'])
print(custom_report)
print("="*50)
print("\n" + "="*50)
print("VGG16 Model Classification Report")
vgg_report = classification_report(true_labels, vgg_preds, target_names=['Cat', 'Dog'])
print(vgg_report)
print("="*50)
==================================================
Custom Model Classification Report
precision recall f1-score support
Cat 0.70 0.76 0.73 1000
Dog 0.74 0.68 0.71 1000
accuracy 0.72 2000
macro avg 0.72 0.72 0.72 2000
weighted avg 0.72 0.72 0.72 2000
==================================================
==================================================
VGG16 Model Classification Report
precision recall f1-score support
Cat 0.89 0.89 0.89 1000
Dog 0.89 0.90 0.89 1000
accuracy 0.89 2000
macro avg 0.89 0.89 0.89 2000
weighted avg 0.89 0.89 0.89 2000
==================================================
4.4 Precision-Recall Curve¶
In [ ]:
custom_probs = custom_model.predict(test_generator)
vgg_probs = vgg_model.predict(test_generator)
custom_precision, custom_recall, _ = precision_recall_curve(true_labels, custom_probs)
vgg_precision, vgg_recall, _ = precision_recall_curve(true_labels, vgg_probs)
custom_auc_score = auc(custom_recall, custom_precision)
vgg_auc_score = auc(vgg_recall, vgg_precision)
plt.figure(figsize=(10, 7))
plt.plot(custom_recall, custom_precision, label=f"Custom Model (AUC={custom_auc_score:.2f})",
color='dodgerblue', linestyle='-', linewidth=2, alpha=0.7)
plt.plot(vgg_recall, vgg_precision, label=f"VGG16 Model (AUC={vgg_auc_score:.2f})",
color='forestgreen', linestyle='--', linewidth=2, alpha=0.7)
plt.axhline(y=0.5, color='darkred', linestyle='-.', linewidth=1.5, alpha=0.6, label='Random Baseline')
plt.title("Precision-Recall Curve Comparison", fontsize=16, fontweight='bold')
plt.xlabel("Recall", fontsize=14)
plt.ylabel("Precision", fontsize=14)
plt.legend(loc='lower left', fontsize=12)
plt.grid(True, alpha=0.3)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.show()
63/63 ━━━━━━━━━━━━━━━━━━━━ 5s 72ms/step 63/63 ━━━━━━━━━━━━━━━━━━━━ 80s 1s/step
4.5 Explore specific examples in which the model failed to predict correctly.¶
In [ ]:
def visualize_misclassifications(model, predictions, generator, model_name):
incorrect_indices = np.where(predictions.flatten() != generator.classes)[0]
print(f"\n{model_name}: Total incorrect predictions = {len(incorrect_indices)}.\n")
# Set up the plot to show the first 5 misclassified images
plt.figure(figsize=(15, 5))
for i, idx in enumerate(incorrect_indices[:5]):
img_path = generator.filepaths[idx]
img = load_img(img_path, target_size=(IMG_HEIGHT, IMG_WIDTH))
plt.subplot(1, 5, i + 1)
plt.imshow(img)
plt.axis('off')
true_label = 'Dog' if generator.classes[idx] == 1 else 'Cat'
pred_label = 'Dog' if predictions[idx] == 1 else 'Cat'
plt.title(f"True: {true_label}\nPred: {pred_label}", fontsize=12)
plt.tight_layout()
plt.show()
visualize_misclassifications(custom_model, custom_preds, test_generator, "Custom Model")
visualize_misclassifications(vgg_model, vgg_preds, test_generator, "VGG16 Model")
Custom Model: Total incorrect predictions = 565.
VGG16 Model: Total incorrect predictions = 215.
Conclusions:¶
Model Performance Overview
- VGG16 Model: The VGG16 model demonstrated exceptional performance across all metrics, including AUC, F1-score, recall, accuracy, and precision. This highlights the effectiveness of using pre-trained models, particularly in complex image classification tasks, thanks to the power of transfer learning.
- Custom Neural Network: Although the custom model performed reasonably well, it was outperformed by VGG16 due to its simpler architecture. The custom model struggled with recognizing intricate patterns that the VGG16 model handled with ease.
Performance Highlights
- Higher Accuracy: VGG16 consistently achieved higher accuracy, showing better generalization on test data and proving its robustness in unseen environments.
- Precision and Recall: VGG16 outperformed the custom model in terms of precision and recall, maintaining better precision at varying recall levels, as evidenced by its higher AUC score.
- Confusion Matrix Insights: The VGG16 model showed fewer misclassifications, indicating its superior ability to distinguish between the two classes (cats and dogs), thus demonstrating its precision in classification tasks.
Error and Failure Analysis
- Both models struggled with certain complex scenarios, such as images with cluttered backgrounds, partially hidden objects, or poor image quality.
- However, the VGG16 model demonstrated a clear advantage by making fewer misclassifications in such challenging cases, indicating its higher robustness.
Improvement Strategies for the Custom Neural Network
- To enhance the performance of the custom model, consider the following:
- Expand Model Complexity: Increase the number of convolutional layers or use larger filter sizes to improve feature extraction.
- Advanced Data Augmentation: Implement more sophisticated data augmentation techniques, such as random rotations, shifts, and zooms, to make the model more resilient.
- Pre-trained Embeddings: Utilize embeddings from pre-trained models to enhance feature extraction and improve the model’s capacity to learn complex patterns.
Fine-Tuning the VGG16 Model
- To further enhance VGG16’s performance, especially in specialized applications:
- Unfreeze Additional Layers: Unlock more layers of the VGG16 model to enable further fine-tuning, improving performance for specific datasets.
- Lower Learning Rate: Implement a lower learning rate when retraining the model to fine-tune its parameters without overshooting the optimal values.